package edu.columbia.gate.plugins;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import gate.Annotation;
import gate.AnnotationSet;
import gate.Resource;
import gate.creole.ResourceInstantiationException;
import gate.creole.gazetteer.AbstractGazetteer;
import gate.creole.gazetteer.GazetteerList;
import gate.creole.gazetteer.GazetteerNode;
import gate.creole.gazetteer.LinearDefinition;
import gate.creole.gazetteer.LinearNode;
import gate.creole.gazetteer.Lookup;
import gate.creole.gazetteer.MappingNode;
import gate.util.Strings;


/**
 * Base class for all edit distance gazetteers. 
 * Defines properties and methods common to all edit distance gazetteers.
 * @author diego
 * @see EditDistanceGazetteer, EditDistanceFullTextGazetteer 
 */
public abstract class AbstractEditDistanceGazetteer extends AbstractGazetteer
{

	/** Debug flag*/
	private static final boolean DEBUG = false;

	public static final String DEF_GAZ_DOCUMENT_PARAMETER_NAME = "document";

	public static final String DEF_GAZ_ANNOT_SET_PARAMETER_NAME = "annotationSetName";

	public static final String DEF_GAZ_LISTS_URL_PARAMETER_NAME = "listsURL";

	public static final String DEF_GAZ_ENCODING_PARAMETER_NAME = "encoding";

	public static final String DEF_GAZ_MAX_EDIT_DISTANCE_PARAMETER_NAME = "maxEditDistance";

	public static final String DEF_GAZ_LONGEST_MATCH_ONLY_PARAMETER_NAME = "longestMatchOnly";

	public static final String DEF_GAZ_FEATURE_SEPARATOR_PARAMETER_NAME = "gazetteerFeatureSeparator";

	public static final String FEATURE_NAME_EDIT_DISTANCE = "editDistance";

	public static final String FEATURE_NAME_ENTRY_MATCHED = "entryMatched";

	public static final String TRANSD_ANNOT_TYPES_PARAMETER_NAME = "annotationTypes";
	
	/**
	 * A gazetteer entry may be followed by a gazetteerFeatureSeparator (default="|" (pipe)) 
	 * and then a name-value pair of the form featureName=featureValue.
	 * We use t=TRANSLATION to provide a translation for the word, in case the gazetteer entry
	 * is itself misspelled on purpose to improve the recall of the gazetteer. 
	 */
	protected static final String GAZETTEER_NODE_TRANSLATION_FEATURE_KEY_NAME = "t"; 

	/** The separator used for gazetteer entry features */
	protected String gazetteerFeatureSeparator;
	
	protected String annotationTypes;
	
	protected String annotationSetName;

	protected double maxEditDistance = Double.MAX_VALUE;

	protected int minEntryLength = Integer.MAX_VALUE;
	protected int maxEntryLength = 0;
	protected Hashtable<String, MatchEntryData> matchCache;
	
	protected static Hashtable<String, Hashtable<String, MatchEntryData>> globalMatchCache;
	

	/** a map of nodes vs gaz lists */
	protected Map listsByNode;

	protected int listCount = Integer.MAX_VALUE; 
	
	/** The words to be recognized (keys) and their EditDistanceLookup tags (values) **/
	protected Hashtable<String, EditDistanceLookup> vocabulary;

	protected ArrayList<Map.Entry<String, EditDistanceLookup>> sortedVocabulary;

	protected Hashtable<String, Integer> majorTypesPriorities;
	
	
	/** Does the actual loading and parsing of the lists. This method must be
	 * called before the gazetteer can be used
	 */
	public Resource init() throws ResourceInstantiationException
	{
		if (listsURL == null)
		{
			throw new ResourceInstantiationException(
					"No URL provided for gazetteer creation!");
		}
		vocabulary = new Hashtable<String, EditDistanceLookup>();
		if(globalMatchCache == null)
		{
			globalMatchCache = new Hashtable<String, Hashtable<String, MatchEntryData>>();
			matchCache = new Hashtable<String, MatchEntryData>();
		}
		else if(globalMatchCache.contains(listsURL.toString()))
		{
			matchCache = globalMatchCache.get(listsURL.toString()); 
			System.out.println("EditDistanceGazetteer.init - Reusing matchCache.");
		}
		else
		{
			matchCache = new Hashtable<String, MatchEntryData>();
		}
		globalMatchCache.put(listsURL.toString(), matchCache);
		
		majorTypesPriorities = new Hashtable<String, Integer>();
		
		definition = new LinearDefinition();
		definition.setSeparator(Strings.unescape(gazetteerFeatureSeparator));
		definition.setURL(listsURL);
		definition.load();
		int linesCnt = definition.size();
		listsByNode = definition.loadLists();
		Iterator inodes = definition.iterator();
		minEntryLength = Integer.MAX_VALUE;
		maxEntryLength = 0;

		int nodeIdx = 0;
		LinearNode node;
		while (inodes.hasNext())
		{
			node = (LinearNode) inodes.next();
			fireStatusChanged("Reading " + node.toString());
			fireProgressChanged(++nodeIdx * 100 / linesCnt);
			readList(node, true);
		} // while iline
		fireProcessFinished();
		return this;
	}

	/** Reads one list (one file) of phrases
	 *
	 * @param node The list to read
	 * @param add if <b>true</b>. will add the phrases found in the list to the ones
	 *     recognised by this gazetter, if <b>false</b> the phrases found in the
	 *     list will be removed from the list of phrases recognised by this
	 *     gazetteer.
	 */
	protected void readList(LinearNode node, boolean add)
			throws ResourceInstantiationException
	{
		String listName, majorType, minorType, languages;
		if (null == node)
		{
			throw new ResourceInstantiationException(
					" LinearNode node is null ");
		}

		listName = node.getList();
		majorType = node.getMajorType();
		minorType = node.getMinorType();
		languages = node.getLanguage();
		GazetteerList gazList = (GazetteerList) listsByNode.get(node);
		if (null == gazList)
		{
			throw new ResourceInstantiationException(
					"gazetteer list not found by node");
		}

		majorTypesPriorities.put(majorType, listCount);
		listCount--;
		
		Iterator iline = gazList.iterator();

		// create default lookup for entries with no arbitrary features
		EditDistanceLookup defaultLookup = new EditDistanceLookup(listName,
				majorType, minorType, languages);

		defaultLookup.list = node.getList();
		if (null != mappingDefinition)
		{
			MappingNode mnode = mappingDefinition
					.getNodeByList(defaultLookup.list);
			if (null != mnode)
			{
				defaultLookup.oClass = mnode.getClassID();
				defaultLookup.ontology = mnode.getOntologyID();
			}
		}//if mapping def

		EditDistanceLookup lookup;
		String entry; // the actual gazetteer entry text
		while (iline.hasNext())
		{
			GazetteerNode gazNode = (GazetteerNode) iline.next();
			entry = gazNode.getEntry().trim().toUpperCase().replaceAll("\\s+",
					" ");

			if (!entry.equals(""))
			{
				if (minEntryLength > entry.length())
					minEntryLength = entry.length();
				if (maxEntryLength < entry.length())
					maxEntryLength = entry.length();

				Map features = gazNode.getFeatureMap();
				if (features == null)
					features = new HashMap<String, Object>();

				String translation = ((String) features.get(GAZETTEER_NODE_TRANSLATION_FEATURE_KEY_NAME));
				if(translation == null || translation.isEmpty()) 
					translation = entry;
				
				features.put(FEATURE_NAME_ENTRY_MATCHED, translation);

				// create a new EditDistanceLookup object with features
				lookup = new EditDistanceLookup(listName, majorType, minorType,
						languages, translation, entry.contains(" "));
				
				lookup.list = node.getList();
				
				if (null != mappingDefinition)
				{
					MappingNode mnode = mappingDefinition
							.getNodeByList(lookup.list);
					if (null != mnode)
					{
						lookup.oClass = mnode.getClassID();
						lookup.ontology = mnode.getOntologyID();
					}
				}// if mapping def
				lookup.features = features;

				if (add)
					this.add(entry, lookup);
				else
					this.remove(entry);
			}
		}
	} // void readList(String listDesc)

	/** Gets a sorted list of unigrams contained in the document, 
	 * and annotated with one of the annotations passed in the annotationSet parameter.
	 * 
	 * @param annotationSet The set of annotations that text to be looked up
	 * must be annotated with. If a unigram is not annotated with an annotation
	 * found in this set, it is skipped.
	 * @return The list of unigrams to be looked up by the gazetteer. 
	 */
	protected ArrayList<Unigram> getUnigrams(AnnotationSet annotationSet)
	{
		String content = document.getContent().toString().toUpperCase();
		ArrayList<TextSegment> segments = new ArrayList<TextSegment>(annotationSet.size());

		TextSegmentComparator segmentComparator = new TextSegmentComparator();
		for(Annotation annot : annotationSet)
		{
			if(this.annotationTypes != null 
					&& this.annotationTypes.contains(annot.getType()))
			{
				TextSegment segment = new TextSegment(annot.getStartNode().getOffset().intValue(), annot.getEndNode().getOffset().intValue());
				int index = Collections.binarySearch(segments, segment, segmentComparator);
				if(index < 0)
					segments.add(-index - 1, segment);
			}
		}
		
		ArrayList<Unigram> unigrams = new ArrayList<Unigram>();
		if(segments.size() <= 0) 
		{
			fireProcessFinished();
			fireStatusChanged("Look-up complete. No annotation with name in '" +  annotationTypes + "' found.");
			return unigrams;
		}

		// Get all words formed by letters, digits and !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~
		// Pattern p = Pattern.compile("[\\p{Graph}]+");
		// Get all words formed by letters, digits and "&"
		Pattern p = Pattern.compile("[\\w\\&]+");
		Matcher m = p.matcher(content);

		String currentWord;

		TextSegment segment = new TextSegment(0, 0);
		int segmentCount = segments.size() ;
		while (m.find())
		{
			segment.start = m.start();
			segment.end = m.end();
			int index = Collections.binarySearch(segments, segment, segmentComparator);
			if(index < 0) index = -index - 1;
			if((index > 0 
					&& segment.start <= segments.get(index - 1).end)
				|| (index < segmentCount
						&& segment.end >= segments.get(index).start))
			{
				currentWord = content.substring(segment.start, segment.end);
				unigrams.add(new Unigram(currentWord, segment.start, segment.end));
			}
		}
		
		return unigrams;
	}
	
	protected class TextSegment
	{
		int start;
		int end;
		public TextSegment(int start, int end)
		{
			this.start = start;
			this.end = end;
		}

	}
	protected class TextSegmentComparator implements Comparator<TextSegment>
	{
		@Override
		public int compare(TextSegment o1, TextSegment o2)
		{
			if(o1.start < o2.start || (o1.start == o2.start && o1.end < o2.end))
				return 1;
			else if (o1.start > o2.start  || (o1.start == o2.start && o1.end > o2.end))
				return -1;
			else
				return 0;
		}
		
	}
	protected class Unigram
	{
		public String word;
		public int start;
		public int end;

		public Unigram(String word, int start, int end)
		{
			this.word = word;
			this.start = start;
			this.end = end;
		}
	}
	
	
	protected class LookupComparator implements Comparator<Map.Entry<String, EditDistanceLookup>>
	{

		@Override
		public int compare(Map.Entry<String, EditDistanceLookup> o1, Map.Entry<String, EditDistanceLookup> o2)
		{
			if(o1.getKey().length() < o2.getKey().length())
				return 1;
			else if (o1.getKey().length() > o2.getKey().length())
				return -1;
			else
				return 0;
		}
		
	}
	
	protected class EditDistanceLookup extends Lookup implements Comparable<EditDistanceLookup>
	{
		private static final long serialVersionUID = 1L;
		public boolean isPhrase = false;
		public String translation;

		EditDistanceLookup(String theList, String major, String minor,
				String theLanguages, String translation, boolean isPhrase)
		{
			super(theList, major, minor, theLanguages);
			this.isPhrase = isPhrase;
			this.translation = translation;
		}

		EditDistanceLookup(Lookup lookup)
		{
			super(lookup.list, lookup.majorType, lookup.minorType,
					lookup.languages);
			this.features = lookup.features;
			if(lookup  instanceof EditDistanceLookup)
			{
				this.isPhrase = ((EditDistanceLookup) lookup).isPhrase;
				this.translation = ((EditDistanceLookup) lookup).translation;
			}
		}

		EditDistanceLookup(String theList, String major, String minor,
				String theLanguages)
		{
			super(theList, major, minor, theLanguages);
		}

		public EditDistanceLookup clone()
		{
			return new EditDistanceLookup(this); 
		}

		@Override
		public int compareTo(EditDistanceLookup o)
		{
			if((this.translation == null && o.translation == null)
					|| (this.translation != null && o.translation != null
							&& this.translation.equals(o.translation))) return 0;
			return 1;
		}
	}

	public String getAnnotationTypes() {
		return annotationTypes;
	}

	public void setAnnotationTypes(String annotationTypes) {
		this.annotationTypes = annotationTypes;
	}

	public String getAnnotationSetName() {
		return annotationSetName;
	}

	public void setAnnotationSetName(String annotationSetName) {
		this.annotationSetName = annotationSetName;
	}
	
	protected class MatchEntryData
	{
		public EditDistanceLookup lookup;
		public String targetWord;
		public String sourceWord;
		public double dlDistance;

		public MatchEntryData(String sourceWord)
		{
			this();
			this.sourceWord = sourceWord;
		}

		public MatchEntryData()
		{
			reset();
		}

		public void copyTo(MatchEntryData match)
		{
			match.sourceWord = this.sourceWord;
			match.lookup = (EditDistanceLookup) this.lookup;
			match.targetWord = this.targetWord;
			match.dlDistance = this.dlDistance;
		}

		public MatchEntryData clone()
		{
			MatchEntryData match = new MatchEntryData();
			this.copyTo(match);
			return match;
		}

		public void reset()
		{
			targetWord = "";
			sourceWord = "";
			dlDistance = 0;
			lookup = null;
		}

		public double getEditDistance()
		{
//			return ((double) this.dlDistance)
//					/ ((double) this.targetWord.length());
			double targetWordLength = targetWord == null || targetWord.isEmpty() ? 1 : targetWord.length();
			double wordinessFactor = (targetWord.length() - targetWord.replace(" ", "").length()) * 0.5;
			double sourceWordLength = sourceWord == null || sourceWord.isEmpty() ? 1 : sourceWord.length(); 
//			return ((double) this.dlDistance)
//					/ Math.min(sourceWordLength, targetWordLength);
			return (((double) this.dlDistance) - wordinessFactor) / Math.min(sourceWordLength, targetWordLength);
		}
	}

	public String getGazetteerFeatureSeparator()
	{
		return gazetteerFeatureSeparator;
	}

	public void setGazetteerFeatureSeparator(String gazetteerFeatureSeparator)
	{
		this.gazetteerFeatureSeparator = gazetteerFeatureSeparator;
	}

	
}
